package infiews.crawler;

import infiews.fetch.PageFetchStatus;
import infiews.util.Logger;
import infiews.web.HTMLParser;
import infiews.web.Page;
import infiews.web.Site;

import java.util.ArrayList;
import java.util.List;

public class WebCrawler implements Runnable {

	public WebCrawler(Site site, CrawlerController controller) {
		this.controller = controller;
		this.site = site;
	}

	protected TaskManager getTaskManager(){
		return controller.getTaskManager();
	}
//
//	public Site getSite(){
//		return site;
//	}

	public List<Page> getTaskList(){
		return pagesToProcess;
	}


	public void onStart() {

	}

	public void onBeforeExit() {

	}

	public void run() {
		onStart();
		pagesToProcess = new ArrayList<Page>(PAGES_COUNT_TO_PROCESS_PER_CYCLE);
		while (getTaskManager().fillTaskList(PAGES_COUNT_TO_PROCESS_PER_CYCLE, pagesToProcess) > 0) {
			for (Page page : pagesToProcess) {
				if ( !getTaskManager().hasPage(page.getURL()) ) {
					preProcessPage(page);
				}
			}
			pagesToProcess.clear();
		}
	}

	public boolean shouldVisit(Page page) {
		if(page.getDeep() >= site.getDeep()){
			return false;
		}
		String href = page.getURL().toLowerCase();
		return site.getPattern().matcher(href).matches();
	}

	public void visit(Page page) {
		log.info( "Deep: " + page.getDeep() + ", " + Thread.currentThread().getName() + ", URL: " + page.getURL() );
	}

	protected int preProcessPage(Page page){

		int statusCode = Crawler.getFetcher().fetch(page);

		if (statusCode != PageFetchStatus.OK) {
			if (statusCode == PageFetchStatus.PageTooBig) {
				log.error("Page was bigger than max allowed size: " + page.getURL());
			}else{
				log.error("Unknown error while fetching page: " + page.getURL());
			}
			if(getTaskManager() != null)
				getTaskManager().addBad(page);
			else{
				int a = 5;
			}
			return statusCode;
		}

		try {
			if (!page.isBinary()) {
				htmlParser.parse(page);
				if(shouldVisit(page)){
					page.setUsefulContent(htmlParser.getUsefulContent());
				}

				List<Page> toSchedule = new ArrayList<Page>();
				List<String> pageLinks = new ArrayList<String>();

				for(String url : htmlParser.getLinks()){
					Page newPage = new Page(url);
					newPage.setDeep(page.getDeep() + 1);
					
					if (shouldVisit(newPage)
							&& !getTaskManager().hasPage(newPage.getURL())
							&& !getTaskManager().hasTask(newPage)
							&& !pageLinks.contains(newPage.getURL())) {
						pageLinks.add(newPage.getURL());
						toSchedule.add(newPage);
					}
				}

				getTaskManager().addTasks(toSchedule);
				page.setURLs(pageLinks);
			}
			
			if (!getTaskManager().hasPage(page.getURL())) {
				visit(page);
			}
			
			getTaskManager().addPage(page);
		} catch (Exception e) {
			e.printStackTrace();
			log.error(e.getMessage() + ", while processing: " + page.getURL());
		}
		return PROCESS_OK;
	}

	protected Site site;
	protected CrawlerController controller;

	List<Page> pagesToProcess;

	protected Logger log = Crawler.log();
	protected HTMLParser htmlParser = new HTMLParser();

	protected final static int PROCESS_OK = -12;
	public static final int PAGES_COUNT_TO_PROCESS_PER_CYCLE = 50;
}